package org.example.webmagic;

import org.example.dao.CompanyMapper;
import org.example.pojo.Company;
import org.example.utils.MD5Util;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Selectable;

import java.util.ArrayList;
import java.util.List;

@SuppressWarnings("ALL")
@Component("sxsProcessor")
public class SXSProcessor implements PageProcessor {

    /**
     * 最好写到service层
     */
    @Autowired
    private CompanyMapper companyMapper;

    private String indexURL = "https://xiaoyuan.shixisheng.com";

    @Override
    public void process(Page page) {
        List<Selectable> nodes = page.getHtml().xpath("//table/tbody/tr[@class='com-table']").nodes();
        //表示结束
        if(nodes.size() < 1) return;

        ArrayList companies = new ArrayList<Company>();

        for (Selectable node:nodes) {
//            System.out.println(">>>>>>>>>>>>>>>>>>>>>>>>>");
//            System.out.println(s.get());
            //先创建一个Company对象
            Company company = new Company();

            //解析网申时间
            String deployTime = node.xpath("//td[@class='list-three']/p/text()").toString();
            company.setDeploy_name(deployTime);

            //网申入口URL
            String companyUrl = node.xpath("//td[@class='list-three']/a/@href").get();
            String url = indexURL + companyUrl;
            company.setCompany_url(url);

            List <String> companyInfo = node.xpath("//td[@class='list-two']/p").all();

            //公司名称
            String companyName = companyInfo.get(0);
            String cpName = companyName.substring(companyName.indexOf("<span>")+6,companyName.indexOf("</span" +
                    ">"));
            company.setName(cpName);

            //出现重复数据即要停止爬虫  (md5)
            String encrypt = MD5Util.encrypt(cpName + deployTime);
            List<String> allMD5 = companyMapper.allMD5();
            if(allMD5.contains(encrypt)) {
                //爬取到重复数据，不需要继续爬取
                break;
//                continue;     不可取
//                return;  这种错误
            }

            //岗位
            String workTypes = companyInfo.get(1);
            String workType = workTypes.substring(workTypes.indexOf("</span>")+7,workTypes.lastIndexOf("</p>"));
            company.setWork_type(workType);

            //工作城市
            String workCitys = companyInfo.get(2);
            String workCity = workCitys.substring(workCitys.indexOf("</span>")+7,workCitys.lastIndexOf("</p>"));
            company.setWork_city(workCity);
            company.setState(0);
            companies.add(company);

            System.out.println("公司名称："+cpName+"工作岗位："+workType+"工作城市："+workCity+"网申入口地址："+url);
        }

        //将数据传给PipeLine，让他存储数据
//        page.getResultItems().put("companies",companies);
        page.putField("companies",companies);

        //表示结束
        if(companies.size() < 10) return;       //处理之后得到的目标数据
//        if(nodes.size() < 10) return;     //页面上所有的数据


        //翻页操作
        //https://xiaoyuan.shixiseng.com/wangshen?p=1&j=opj_aepdep2mgrcj  替换p=XX
        String url = page.getUrl().get();
        Integer num = Integer.valueOf(url.substring(url.indexOf("p=")+2,url.indexOf("&")));
        num = num+1;
        url = url.replace((url.substring(url.indexOf("p="),url.indexOf("&"))),"p="+String.valueOf(num));

        //页面跳转
        page.addTargetRequest(url);
    }

    @Override
    public Site getSite() {
        Site site = Site.me();
        site.setRetryTimes(3);  //设置重试次数
        site.setSleepTime(1000);    //睡眠时间，模拟正常访问
        //对于防爬虫技术较好的网站需要cookie
        site.addCookie("Cookie","SXS_XSESSION_ID=\"2|1:0|10:1616241900|15:SXS_XSESSION_ID|48:OTA0ODcyYmQtZWE5NC00NDVlLTk1NTUtZjM3OWJiOTMyMTFm|1047bb8de25057b8d6904098643ce3d5079383a62cd6fbe57ac180d63bf0918d\"; SXS_XSESSION_ID_EXP=\"2|1:0|10:1616241900|19:SXS_XSESSION_ID_EXP|16:MTYxNjMyODMwMA==|fa3b381c3ade5d989fe1c8afb3997be4538cc5c5e4d9545388464de6313da751\"; __jsluid_s=bc9f6281b4ee1afcee9ab3b7d8e6f4cb; Hm_lvt_e04d7e6abe97e1a0a12089035e2762a8=1616241901; Hm_lpvt_e04d7e6abe97e1a0a12089035e2762a8=1616241979; utm_source_first=PC; utm_source=PC; …ross=%7B%22distinct_id%22%3A%221784f87770bbd-0691858541f9d4-4c3f227c-1049088-1784f87770c81%22%2C%22%24device_id%22%3A%221784f87770bbd-0691858541f9d4-4c3f227c-1049088-1784f87770c81%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_referrer%22%3A%22%22%2C%22%24latest_referrer_host%22%3A%22%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%7D%7D; sajssdk_2015_cross_new_user=1");
        return site;
    }
}
